package com.qf.program.service;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.RandomUtil;
import cn.hutool.core.util.ZipUtil;
import cn.hutool.db.Db;
import cn.hutool.db.Entity;
import cn.hutool.db.ds.DSFactory;
import com.qf.program.utils.http.HttpClientConfig;
import com.qf.program.utils.http.HttpClientUtil;
import com.qf.utils.SmallTool;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.message.BasicNameValuePair;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;
import org.jsoup.select.Elements;

import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicInteger;

@Slf4j
public class PubmedXmlLocalCheck {

    public static boolean checkExist = false;


    public static void main(String[] args) throws SQLException {


        String insert = TableConfig.PMXMLTABLEALLNEW;
        //是不是要检查本地有没有 有的话默认不插入
        boolean isCheck = true;
        // 开启本地代理
        boolean isProxy = false;
        Db use = Db.use();

        BasicCookieStore cookieStore = new BasicCookieStore();
        List<String> ips = new ArrayList<>();
        List<String> ipsMy = new ArrayList<>();
        String sp = "<=";
        if (FileUtil.exist("/home/ip.txt")) {
            ips = FileUtil.readLines("/home/ip.txt", "utf-8");
            log.info("共获取到" + ips.size() + "个IP地址");

            log.info("sp+++" + sp);
        }

        String xmlIdsTable = "xml_ids";
        update(use, xmlIdsTable);

        //批量把剩余错误的进行重试修改数据
        if (ips.size() == 0) {
            if (FileUtil.exist("/usr/local/gcr/proxyip.txt")) {
                ipsMy = FileUtil.readUtf8Lines("/usr/local/gcr/proxyip.txt");
                log.info("共获取到" + ipsMy.size() + "个IP地址");
            }
        }
        if (!isProxy) {
            ipsMy = new ArrayList<>();
            ips = new ArrayList<>();

        }

        while (true) {
            List<Entity> list = use.query("SELECT * FROM " + xmlIdsTable + " WHERE flag = -1  LIMIT 100");
            if (list.size() == 0) {
                break;
            }
            //防止被封 本地单线程
            SmallTool.SleepMillis(1600L);
            List<String> finalIps = ips;
            AtomicInteger effectiveId = new AtomicInteger();

            List<String> finalIpsMy = ipsMy;
            list.forEach(j -> {
                Integer id = j.getInt("id");
                log.info("获取ID：id ++++ [ " + id + "  ids from++++" + j.getInt("fromid") + "-" + j.getInt("toid") + "]的XML数据中……");
                try {
                    Integer flag = 0;
                    String pmids = j.getStr("pmids");
                    String replace = pmids.replace("[", "").replace("]", "");
                    String res = "";
                    if (checkExist) {
                        res = PubmedXmlLocal.checkIsDown(insert, use, replace);
                    } else {
                        res = replace;
                    }
                    if (StringUtils.isEmpty(res)) {
                        SmallTool.printMess("数量一致 略过+++++++++++++");
                        flag = 1;
                    } else {
                        SmallTool.printMess("检查后需要下载的pmid++" + res);
                        HttpClientConfig config = new HttpClientConfig()
                                .setUrl("https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi")
                                .build();
                        if (finalIps.size() > 0) {
                            config.setLocalIp(finalIps.get(RandomUtil.randomInt(0, finalIps.size() - 1)));
                            log.info("使用IP地址：" + config.getLocalIp());
                        }
                        if (!isProxy) {
                            if (finalIpsMy.size() > 0) {
                                String proxyip = finalIpsMy.get(RandomUtil.randomInt(0, finalIpsMy.size() - 1));
                                log.info("使用的ip地址++" + proxyip);
                                String[] split = proxyip.split(":");
                                config.setProxyIp(split[0]);
                                config.setProxyPort(Integer.parseInt(split[1]));
                            }
                        } else {
                            config.setProxy(isProxy);
                        }
                        config.setSslUse(true);
                        config.setCookieStore(cookieStore);
                        config.getNvpsList().add(new BasicNameValuePair("db", "pubmed"));
                        config.getNvpsList().add(new BasicNameValuePair("retmode", "xml"));
                        config.getNvpsList().add(new BasicNameValuePair("id", res));
                        String body = HttpClientUtil.postXmlPm(config);

                        try {
                            if (body != null && body.equals("123")) {
                                SmallTool.printMess("gcr123");
                                FileUtil.appendUtf8String("id 异常重新下载在++" + id + "\n", "/usr/local/gcr/20230208.log");
                                flag = 1;
                            } else {
                                Document parse = Jsoup.parse(body, "", Parser.xmlParser());
                                Elements pubmedArticle = parse.select("PubmedArticleSet");
                                for (Element element : pubmedArticle) {
                                    Elements children = element.children();
                                    if (children.size() > 0 || children.size() == 0) {
                                        flag = 1;
                                        effectiveId.getAndIncrement();
                                    }
                                    if (children.size() == 0) {
                                        log.info("注意  单线程获取数据拿到是空 id 是+" + id + "ids +" + res);
                                    }
//                                        log.info("共获取到[" + children.size() + "]条数据，开始插入数据库……");
                                    Integer oknum = 0;
                                    for (Element child : children) {
                                        try {
                                            String pmid = child.select("PMID").first().text();
                                            String xml = new StringBuffer().append("<" + child.nodeName() + ">\n").append(child.html()).append("\n</" + child.nodeName() + ">").toString();
                                            Long time = System.currentTimeMillis();
                                            Entity ins = Entity.create(insert)
                                                    .set("pmid", pmid)
                                                    .set("content", ZipUtil.gzip(xml, "utf-8"))
                                                    .set("lastupdate", time);
//                                        use.insert(ins);
                                            use.insertOrUpdate(ins, "pmid");
                                            oknum++;
                                        } catch (Exception ex) {
                                            log.info("查看异常的字符串" + ex.toString());
                                        }
                                    }
                                    SmallTool.printMess("插入数据完成，共插入[" + oknum + "]条数据");

                                }
                            }

                        } catch (Exception ex) {
                            ex.printStackTrace();
                        }

                    }


                    if (flag == 1) {
                        try {
                            use.execute("UPDATE  " + xmlIdsTable + "  SET flag = ? WHERE id = ?", 1, j.getInt("id"));
                        } catch (SQLException throwables) {
                            throwables.printStackTrace();
                        }
                    } else {
                        try {
                            use.execute("UPDATE  " + xmlIdsTable + "  SET flag = ? WHERE id = ?", -1, j.getInt("id"));
                        } catch (SQLException throwables) {
                            throwables.printStackTrace();
                        }
                    }
                } catch (Exception ex) {
                    try {
                        use.execute("UPDATE  " + xmlIdsTable + "  SET flag = ? WHERE id = ?", -1, j.getInt("id"));
                    } catch (SQLException throwables) {
                        throwables.printStackTrace();
                    }
                    ex.printStackTrace();
                }
            });
            SmallTool.printMess("全部有效插入数量是+++" + effectiveId.get());
        }
        boolean b = updateCheck();
        if (b) {
            SmallTool.printMessError("进行新的测试");
            PubmedXmlLocalCheck.main(args);
        }

        合并到本地256G.main(args);

    }

    @SneakyThrows
    public static void update(Db use, String table) {
        List<String> strings = new ArrayList<>();
        try {
            strings = FileUtil.readUtf8Lines("/usr/local/gcr/20230208.log");
        } catch (Exception e) {
            e.printStackTrace();
        }

        List<String> allids = new ArrayList<>();
        if (strings.size() > 0) {
            for (String string : strings) {
                String ID = string.split("\\++")[1];
                allids.add(ID);
            }
        }
        if (allids.size() > 0) {
            String join = CollUtil.join(allids, ",");
            String sqlUp = "update " + table + " set flag = -1 where id in ( " + join + " )";
            int execute = use.execute(sqlUp);
            SmallTool.printMess("修正了数据大小++" + execute);
        }
        FileUtil.writeUtf8String("", "/usr/local/gcr/20230208.log");
    }

    @SneakyThrows
    public static boolean updateCheck() {
        List<String> strings = FileUtil.readUtf8Lines("/usr/local/gcr/20230208.log");
        if (strings.size() > 0) {
            return true;
        } else {
            return false;
        }

    }


}
